from proxypool.stroages import RedisClient
import time
from pyquery import PyQuery as pq
from lxml import etree
import requests

base_headers  =  {
    'User-Agent':  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
    # 'Accept-Encoding':  'gzip, deflate, sdch',
    # 'Accept-Language':  'zh-CN,zh;q=0.8'
}

def get_page(url):
    headers = dict(base_headers)
    try:
        r = requests.get(url, headers=headers)
        print('Getting result',url,r.status_code)
        if r.status_code == 200:
            return r.text
    except ConnectionError:
        print('Crawling Failed', url)
        return None

class ProxyMetaclass(type):
    def __new__(cls,name,bases,attrs): 
        count = 0
        attrs['__CrawlFunc__'] = []
        for k, v in attrs.items():
            if 'crawl' in k:
                attrs['__CrawlFunc__'].append(k)
                count += 1
        attrs['__CrawlFuncCount__'] = count
        return type.__new__(cls,name,bases,attrs)

class Crawler(object, metaclass=ProxyMetaclass):
    def get_proxies(self,callback):
        proxies = []
        for proxy in eval("self.{}()".format(callback)):
            print('成功获取到代理',proxy)
            proxies.append(proxy)
        return proxies

    def crawl_xdl(self,page_count=2):
        start_url = 'http://www.xiladaili.com/gaoni/{}'
        urls = [start_url.format(page) for page in range(1,page_count+1)]
        for url in urls:
            print('Crawling', url)
            html = get_page(url)
            if html:
                etree_html = etree.HTML(html)
                ip_ports = etree_html.xpath("//tbody/tr/td[1]/text()")
                for ip_port in ip_ports:
                    host = ip_port.partition(":")[0]
                    port = ip_port.partition(":")[2]
                    yield ":".join([host,port])
                time.sleep(2)

POOL_UPPER_THRESHOLD = 10000
class Getter():
    def __init__(self):
        self.redis = RedisClient()
        self.crawler = Crawler()

    def is_over_threshold(self):
        if self.redis.count() >= POOL_UPPER_THRESHOLD:
            return True
        else:
            return False
    
    def run(self):
        print('获取器开始执行')
        if not self.is_over_threshold():
            for callback_label in range(self.crawler.__CrawlFuncCount__):
                callback = self.crawler.__CrawlFunc__[callback_label]
                proxies = self.crawler.get_proxies(callback)
                for proxy in proxies:
                    self.redis.add(proxy)


if __name__ == '__main__':
    getter = Getter()
    getter.run()